package us.codecraft.webmagic.main;

import java.io.File;
import java.io.FileInputStream;
import java.io.FileNotFoundException;
import java.io.IOException;
import java.util.Properties;

import us.codecraft.webmagic.Spider;
import us.codecraft.webmagic.pipeline.FilePipeline2;
import us.codecraft.webmagic.pipeline.HbasePipeline;
import us.codecraft.webmagic.pipeline.MongoPipeline2;
import us.codecraft.webmagic.processor.PartOfPageProcessor;
import us.codecraft.webmagic.processor.PureProcessor;
import us.codecraft.webmagic.scheduler.MongoScheduler;


public class JCrawl {
	static String starting_url;
	static String url_pattern;
	static int threads;
	static String mongodb;
	static String mongocollection;
	static String store;
	static String hbase_table;
	static String mongohost;
	static String schedulerdb;
	static String schedulercoll;
	static String classEl;
	/**
	 * @param args
	 * @throws Exception 
	 */
	public static void main(String[] args) throws Exception {
		// TODO Auto-generated method stub
		
		Config();
		Spider me = null;
		if (store.equals("mongo"))  me = Spider.create(new PureProcessor(starting_url)).
				pipeline(new MongoPipeline2(mongohost,mongodb,mongocollection)).
				scheduler(new MongoScheduler(mongohost,schedulerdb,schedulercoll)).
				thread(threads);
		if (store.equals("hbase"))	me = Spider.create(new PureProcessor(starting_url)).
				pipeline(new HbasePipeline(hbase_table)).
				thread(threads);
		
		me = Spider.create(new PartOfPageProcessor(starting_url,classEl)).
				pipeline(new FilePipeline2()).
				thread(threads);
		
		me.run();
	}
	
	private static void Config () {
		Properties p = new Properties();
		try {
			p.load(new FileInputStream(new File("./config")));
		} catch (FileNotFoundException e) {
			// TODO Auto-generated catch block
			e.printStackTrace();
		} catch (IOException e) {
			// TODO Auto-generated catch block
			e.printStackTrace();
		}
		
		starting_url = p.getProperty("starting_url");
		url_pattern = p.getProperty("url_pattern");
		mongodb = p.getProperty("mongodb");
		mongocollection = p.getProperty("mongocollection");
		mongohost = p.getProperty("mongohost");
		schedulerdb = p.getProperty("schedulerdb");
		schedulercoll = p.getProperty("schedulercoll");
		
		threads = Integer.parseInt(p.getProperty("threads"));
		store = p.getProperty("store");
		hbase_table = p.getProperty("hbase_table");
		
		classEl = p.getProperty("classEl");
	}

}
